package com.bigdata.spark.projectone

import java.net.URL

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @author Gerry chan
 * @version 1.0
 *  案例实现-求用户访问学科的子网top3
 */
object ObjectCount1 {
  def main(args: Array[String]): Unit = {
    val conf:SparkConf = new SparkConf().setAppName("ObjectOUcnt1").setMaster("local[2]")
    val sc:SparkContext = new SparkContext(conf)

    //获取数据
    val file:RDD[String] = sc.textFile("datas/access.txt")
    //提取url并生成元组
    val urlAndOne:RDD[(String,Int)] = file.map(line=> {
      val fileds = line.split("\t")
      val url = fileds(1)
      (url,1)
    })
    //把相同的Url聚合
    val sumedUrl:RDD[(String, Int)] = urlAndOne.reduceByKey(_+_)

    //获取学科信息
    //ArrayBuffer((php.itcast.cn,http://php.itcast.cn/php/course.shtml,459), (java.itcast.cn,http://java.itcast.cn/java/course/base.shtml,543),)
    val project:RDD[(String, String, Int)] = sumedUrl.map(x=> {
      val url = x._1  //url
      val count = x._2 //请求的url的次数
      val project = new URL(url).getHost
      (project, url, count)
    })

    //用学科来分组，聚合后得到结果
    val res:RDD[(String,List[(String, String, Int)])] =
      project.groupBy(_._1).mapValues(_.toList.sortBy(_._3).reverse.take(3))
    println(res.collect().toBuffer)
    sc.stop()
  }

}
